from pyppeteer import launch
from pyppeteer.browser import Browser
from pyppeteer.network_manager import Response
from urllib.parse import quote
from lxml import etree
from mini_tools.cjjtool import *
from mini_tools.myexcel import *
from sql import *
import asyncio,re,openpyxl,shutil,time,traceback

pat_pid=re.compile(r'-i[.](\d+?[.]\d+)[?]?')


ycjp=[]
shopids=[]
prices={}
ms=None
async def intercept_response(res:Response,kw,sernum,p):
    global ycjp,shopids
    req=res._request
    cur_url=req.url
    if "shopee.co.id/api/v4/search/search_items" in cur_url:
        try:
            if (kw,p) in ycjp:
                #print(f'{kw}-{p},已存在')
                return
            ycjp.append((kw,p))
            json_data = await res.json()
            pros=json_data['items']
            print(f'《{kw}》,第{p+1}页,截取到:{len(pros)}个产品')
            isfhcount=0
            for item in pros:

                #tid=item['item_basic']['is_official_shop']
                if not item['adsid']:
                    shopid=str(item['item_basic']['shopid'])
                    if shopid not in shopids:
                        shopids.append(shopid)
                        itemid=item['item_basic']['itemid']
                        cur_purl=f'https://shopee.co.id/pro-i.{shopid}.{itemid}'
                        aff=ms.ExecNoQuery('insert into [searchUrl]([GuanJianCi],[SouSuoYeChanPinWangZhi],[shopid]) values(?,?,?)',(kw,cur_purl,shopid))
                        if aff:
                            isfhcount+=1

            print(f'《{kw}》,第{p+1}页,接口采集完毕,有{isfhcount}个新增链接')
        except Exception as e:
            traceback.print_exc()
            print(f'{kw}-第{p+1}页,接口错误 => {e}')

async def cj_purl(browser:Browser,w,h):
    global dcj
    page = await browser.newPage()
    await page.setViewport({'width':w , 'height': h}) #设置窗口大小
    await page.evaluateOnNewDocument('''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')


    while True:
        if len(dcj)==0:
            print('任务队列为0')
            break
        maxp=9
        p=0
        islastpage= False
        print(f'当前任务队列:{len(dcj)}')

        pitem=dcj.pop(0)
        kid,kw,sernum=pitem
        purl=f'https://shopee.co.id/search?filters=7%2C6&keyword={quote(kw)}&maxPrice=1000000&minPrice=200000&page=0&ratingFilter=4&sortBy=sales'       
        err_cont=0
        while True:

            try:
                page.on('response', lambda rep: asyncio.ensure_future(intercept_response(rep,kw,sernum,p)))
                if p>maxp-1:
                    break
                print(f'《{kw}》,开始采集第{p+1}页')
                if p==0:
                    await repeat_goto(page,purl)
                    await page.waitFor(5000)
                    jccpcz=False
                    while True:
                        noexceit=await page.xpath('//div[@role="main" and @class="product-not-exist"]')
                        elempty= await page.xpath('//div[@role="main"]//div[@class="shopee-search-empty-result-section"]')
                        dlel=await page.xpath('HRbynn')
                        oo=await page.xpath('//a[@data-sqe="link"]')
                        if noexceit or elempty or dlel:
                            print(f'《{kw}》,产品不存在')
                            break
                        if oo:
                            jccpcz=True
                            break
                        await page.waitFor(1000)
                    if not jccpcz:
                        break

                    cot =await page.content()
                    btn=await page.xpath('//a[@class="shopee-button-outline shopee-button-outline--disabled shopee-mini-page-controller__next-btn"]')
                    if btn:
                        islastpage=True
                    elmaxp=etree.HTML(cot).xpath('//span[@class="shopee-mini-page-controller__total"]/text()')
                    if elmaxp:
                        maxp=int(elmaxp[0].strip())

                else:
                    await page.waitFor(5000)
                    jccpcz=False
                    while True:
                        noexceit=await page.xpath('//div[@role="main" and @class="product-not-exist"]')
                        elempty= await page.xpath('//div[@role="main"]//div[@class="shopee-search-empty-result-section"]')
                        dlel=await page.xpath('HRbynn')
                        oo=await page.xpath('//a[@data-sqe="link"]')
                        if noexceit or elempty or dlel:
                            print(f'《{kw}》,产品不存在')
                            break
                        if oo:
                            jccpcz=True
                            break
                    if not jccpcz:
                        break
                    btn=await page.xpath('//a[@class="shopee-button-outline shopee-button-outline--disabled shopee-mini-page-controller__next-btn"]')
                    if btn:
                        islastpage=True
                    else:
                        await page.click('a.shopee-mini-page-controller__next-btn')
                        await page.waitFor(3000)

                p+=1
                if islastpage:
                    break

            except Exception as e:
                print(f'《{kw}》,第{p+1}页-出错 => {e}')
                traceback.print_exc()
                err_cont+=1
                if err_cont>10:
                    break
            
        
        aff2=ms.ExecNoQuery('update [keyword] set [cjstatus]=2 where [id]=?',(kid,))
        print(f'kw:{kw},采集完毕,更改状态{aff2}')


dcj=[]
async def begin_cj_sousuo_purl(w,h):
    global dcj,ms,shopids
    ms = sqlseverDB('{SQL Server}', '199.33.126.84,1433', 'shopeecoid', 'sa', 'Y3GwLHaP8zQ0seZG')

    while True:

        objcig=get_config()
        bargs=['--disable-infobars','--start-maximized']
        if objcig['proxy']:
            bargs.append(f"--proxy-server={objcig['proxy']}")
        kchrome()
        browser = await launch(
            handleSIGINT=False,
            handleSIGTERM=False,
            handleSIGHUP=False,
            headless=objcig['headless'],
            userDataDir=objcig['userDataDir'],
            executablePath=objcig['executablePath'],
            args=bargs
        )
        rshids=ms.ExecQuery('select shopid from searchUrl')
        shopids=[r[0] for r in rshids]

        dcj=ms.ExecQuery('select top 1000 [id],[GuanJianCi],[SouSuoLiang] from [keyword] where [cjstatus]=0 order by [SouSuoLiang] desc')
        if len(dcj)==0:
            print('数据库暂无待采集数据,请稍后重试')
            break
        print(f'成功从数据库获取{len(dcj)}个待采集关键词')

        #更新状态值
        idlist=[str(item[0]) for item in dcj]
        idstr=','.join(idlist)
        aff=ms.ExecNoQuery(f'update [keyword] set [cjstatus]=1 where [id] in ({idstr})')
        print(f'成功将{aff}个关键词采集状态标注为1,避免多开重复获取')
        tasks=[asyncio.create_task(cj_purl(browser,w,h)) for j in range(1)]
        
        await asyncio.gather(*tasks)

        r1=ms.ExecQuery('select count(id) from keyword where cjstatus=0')
        r2=ms.ExecQuery('select count(id) from keyword where cjstatus=1')
        r3=ms.ExecQuery('select count(id) from keyword where cjstatus=2')
        print(f'本次采集完毕,数据库当前状态 => 未采关键词:{r1[0][0]},正在采关键词:{r2[0][0]},已采关键词:{r3[0][0]}')
        await browser.close()
    




